/**
 * 
 */
package ca.uwindsor.cs.deepweb.utility.sougouweb;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;

import ca.uwindsor.cs.deepweb.utility.DocumentRegrouper;
import ca.uwindsor.cs.deepweb.utility.GZipDecompressor;

/**
 * @author Liang Jie
 * 
 */
public class SogouCorpusExtractor {
	private String destinationDirectory;

	protected BufferedReader bufferedreader_in;

	protected GZipDecompressor decompressor;

	protected DocumentRegrouper regrouper;

	protected int total;

	protected int amount;

	public SogouCorpusExtractor(String destdir, int limit) {
		decompressor = new GZipDecompressor(System.getProperty("user.dir"));
		regrouper = new DocumentRegrouper("SogouWeb", "xml");
		destinationDirectory = destdir;
		total = 0;
		amount = limit;
	}

	/**
	 * @param dir
	 */
	public void start(File dir) {
		File[] files = dir.listFiles();
		for (int offset = 0; offset < files.length; offset++) {
			File file = files[offset];
			if (file.isFile()) {
				String filename = file.getName();
				if (filename.endsWith(".gz")) {
					File decompressedfile = null;
					try {
						decompressor.decompress(file);

					} catch (FileNotFoundException e) {
						e.printStackTrace();
						System.exit(-1);
					} catch (IOException e) {
						e.printStackTrace();
						System.exit(-1);
					}
					decompressedfile = decompressor.getDecompressedFile();
					extract(decompressedfile);
					System.gc();
					decompressedfile.delete();
				}
				if (total == amount) {
					break;
				}
			} else if (file.isDirectory()) {
				start(file);
			} else {
				continue;
			}
		}

	}

	/**
	 * @param file
	 */
	public void extract(File file) {
		InputStreamReader reader = null;
		String string_line = new String();
		try {
			reader = new InputStreamReader(new FileInputStream(file), "GBK");
			bufferedreader_in = new BufferedReader(reader);
			string_line = bufferedreader_in.readLine().trim();
			StringBuffer content = new StringBuffer(
					"<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
			content.append("\n");
			while (string_line != null) {
				if (string_line.startsWith("</doc>")) {
					content.append(string_line);
					// save to file
					String newfilename = regrouper.getFilename();
					File dir = new File(destinationDirectory + File.separator
							+ regrouper.getDirectories());
					if (!dir.exists()) {
						dir.mkdirs();
					}
					File newfile = new File(dir, newfilename);
					FileOutputStream fp = new FileOutputStream(newfile);
					fp.write(content.toString().getBytes("UTF-8"));
					fp.flush();
					fp.close();
					total++;

					System.out
							.println(newfile.getCanonicalPath() + " created!");
					
					content = new StringBuffer(
							"<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
				} else {
					content.append(string_line);
					content.append("\n");
				}
				string_line = bufferedreader_in.readLine();
			}
			bufferedreader_in.close();
			reader.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		}
		System.out.println("Total files: " + total);

	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// args = new String[2];
		// args[0] = "D:\\学习资料\\Graduate Study\\Research\\Data\\test";
		// args[1] = "D:\\学习资料\\Graduate Study\\Research\\Data\\extracted";

		if (args.length != 3) {
			System.out
					.println("usage: java SogouCorpusExtractor GZFileDirectory outputDirectory limit");
		} else {
			long startTime = new Date().getTime();
			SogouCorpusExtractor reader = new SogouCorpusExtractor(args[1],
					Integer.parseInt(args[2]));
			reader.start(new File(args[0]));
			long endTime = new Date().getTime();
			System.out.println("It takes " + (endTime - startTime)
					+ " milliseconds to extrat the files from" + args[0]);
		}
	}

}
